#-------------------------------------------------------------------------------
# Name:        module1
# Purpose:
#
# Author:      Kim
#
# Created:     29/10/2011
# Copyright:   (c) Kim 2011
# Licence:     <your licence>
#-------------------------------------------------------------------------------
#!/usr/bin/env python

import json, sys
import preprocessor

input_filenames = ["t_s_1009_1", "t_s_1009_10", "t_s_1009_11", "t_s_1009_12", \
                  "t_s_1009_13", "t_s_1009_14", "t_s_1009_15"]
output_filename = "t_s_1009_1_output.txt"
input_filename = "t_s_1009"

def processOneFiles():
    print "Start processing files \n\n\n"
    file_fp = open(input_filename, "w")
    #Process the files into one file
    for each_file in input_filenames:
        print "Start processing file: " + each_file + "\n"
        test_fp = open(each_file, "r")
        line = test_fp.readline()
        while(line <> ""):
            file_fp.write(line)
            line = test_fp.readline()
    file_fp.close()

def getJSONObjects(path_file):
    count = 0
    #Load the json objects
    print "\nLOG: Attempt to load JSON objects from file: " + path_file + "\n"
    fp = open(path_file, "r")
    if (fp == None):
        print "Failed to open : " + path_file
        sys.exit(1)

    txt_count = 0
    obj_lists = []
    line = fp.readline()
    total_tweets = {}
    while (line <> ""):
        if (count % 100000 == 0):
            print str(count) + " lines read\n\n"
            index = count / 100000
            txtcount, total_tweets = processJSONObjects(obj_lists, index, total_tweets)
            txt_count = txt_count + txtcount
            print total_tweets
            obj_lists = []
        try:
            objs = json.loads(line)
            if (objs == None):
                print "JSON Object Decoding error"
                sys.exit(1)
            obj_lists.append(objs)
        except:
            print "ERROR in JSON Object: " + line + "\n\n"
        line = fp.readline()
        count = count + 1
    print "\nFinished load JSON objects: Total " + str(txt_count) + " Texts"
    fp.close()
    return obj_lists


def processJSONObjects(objects, index, user_tweets):
    pre = preprocessor.Preprocessor()
    language = 0
    output_path = "output_" + str(index) + ".txt"
    key_output_path = "key_output_" + str(index) + ".txt"

    output = open(output_path, "w")
    keys = open(key_output_path, "w")

    if (output == None):
        print "Failed to open : " + output_filename
        sys.exit(1)

    txt_count = 0
    #Get each objects
    for each_obj in objects:
        #Get the user
        if (each_obj.has_key('user') and each_obj.has_key('text')):
            usr_obj = each_obj['user']
            if (usr_obj.has_key('id') and usr_obj.has_key('screen_name')):
                id_key = usr_obj['id']
                output.write(str(id_key))
                output.write("(")
                output.write(usr_obj['screen_name'])
                output.write("): ")
            else:
                print str(usr_obj.keys())+ "\n\n"
                continue

            #Get the text
            try:
                tweet = each_obj['text']
                tweet = pre.Preprocess(tweet)
                output.write(tweet)
                output.write("\n")
                txt_count = txt_count + 1

                #Add it to the memory dictionary
                if (user_tweets.has_key(id_key)):
                    tweets = user_tweets[id_key]
                else:
                    tweets = []
                tweets.append(tweet)
                user_tweets[id_key] = tweets

            except:
                #Do Nothing
                language = language + 1
        else:
            keys.write(str(each_obj.keys()) + "\n")

    print "\nFinished Getting Texts"
    output.close()
    keys.close()

    return txt_count, user_tweets

#Comment this out if you want to process just part of the file
processOneFiles()
objects = getJSONObjects(input_filename)
